{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.57344484,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('MountainCarContinuous-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        reward = -1.0\n",
    "        if done:\n",
    "            reward = 100\n",
    "\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhlUlEQVR4nO3daWwTd/4/8Pc4vnLZuW2CHZJyhftIQmISoEBKyqal0GqPqg+q7qoVbahKW1VqpH+pVlop1e6T3e5W9MGqh3ZXy4pV2W0RpA0J5CDOTSAECKULJBA7CQmxcziO7fn8H7SZHwZKkxAysf15SSPVM1/bH0+Yd2e+M/MdgYgIjDEmA4XcBTDGQhcHEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNrIF0EcffYTU1FRotVpkZ2ejoaFBrlIYYzKRJYD+9a9/4a233sL777+PlpYWrFmzBgUFBejt7ZWjHMaYTAQ5bkbNzs5GVlYW/vKXvwAARFGE2WzG66+/jnfffXe2y2GMyUQ52184Pj6O5uZmFBcXS/MUCgXy8/NhtVrv+x632w232y29FkURAwMDiI+PhyAIj7xmxtjUEBGGhoaQnJwMheLHD7RmPYBu3boFn88Hg8HgN99gMODSpUv3fU9JSQl++9vfzkZ5jLEZ1NXVBZPJ9KPLZz2ApqO4uBhvvfWW9NrhcCAlJQVdXV3Q6XQyVsYYux+n0wmz2Yzo6OgHtpv1AEpISEBYWBh6enr85vf09MBoNN73PRqNBhqN5p75Op2OA4ixOeynukhm/SyYWq1GRkYGysvLpXmiKKK8vBwWi2W2y2GMyUiWQ7C33noLL774IjIzM7Fhwwb88Y9/xMjICF566SU5ymGMyUSWAPrlL3+Jvr4+HDhwAHa7HWvXrkVpaek9HdOMseAmy3VAD8vpdEKv18PhcHAfEGNz0GS30YA4C8YYe/TGx2/C57sNlcqMsLAIAMpHfp0dBxBjDAAwOPgfdHf/PyiVSdBq0xERsQbh4auh1a6ASmVAWJgeE+etZiqYOIAYYz8Q4fMNwucbhNt9GQ7HlwCUUCi0UKvN0GgWITJyA3S6AkRGZs3IN3IAMcZ+hAJKZQyUynkID18h7RGpVMkz9g0cQIwxAIDHo4RWuxbh4ct+CJt10GjSoFIlQ6HQAlDMeJ8QBxBjDADQ0pKIjRv/i7i4+RCEsFn5Th4RkTEGAPB4FBCEqFkLH4ADiDEmIw4gxphsOIAYY7LhAGKMyYYDiDEmGw4gxphsOIAYY7LhAGKMyYYDiDEmGw4gxphs+F4wxthPunPgVCLyuyn1YW5Q5QBijPkhIjidTgwPD8Nms6Gvrw9jY2NwOBwIDw/H8PAwwsPDoVKpoNVqERcXh6ioKMyfPx/R0dFQq9WT/i4OIMZCHBHB4/FgYGAAX3/9NURRhMvlgtFoRGJiIhYtWoTY2FioVCq/xyyLooixsTEMDAzAZrPh6tWrcDgciIyMRGxs7KS+mwOIsRBERBBFETabDWfOnMGtW7eg1WqRkZGBlJQUvweBPugQS6/Xw2AwID09XZo3ODiIjo6OSdXBAcRYiHG73bh8+TKamppgNBqxfv16JCUlQamc/iD0d74vNjYWy5cvn9T7OIAYCwETh1ltbW04f/480tPT8fOf/xxRUVGy1sUBxFiQ8/l8aG9vR1tbG5YtW4bnn38eKpXqkT9yZzI4gBgLUkSEmzdvorGxEQkJCdi9ezciIiLmRPBM4ABiLAiNj4+joaEBPT092LZtG2JiYuZU8EzgAGIsiBARuru7cfLkSWRmZsJisSAsbPbGeJ4qDiDGgoTP50Nrayt6enqwa9cuREdHz8m9njtN+V6wqqoqPP3000hOToYgCPjPf/7jt5yIcODAAcybNw/h4eHIz8/Ht99+69dmYGAAL7zwAnQ6HWJiYvCb3/wGw8PDD/VDGAtlIyMj+PrrryEIAgoKCqDT6eZ8+ADTCKCRkRGsWbMGH3300X2X//73v8eHH36Ijz/+GPX19YiMjERBQQHGxsakNi+88ALa29tRVlaGo0ePoqqqCq+88sr0fwVjIYqI0NXVhRMnTmDdunVYt27dnD7kugc9BAB05MgR6bUoimQ0GukPf/iDNG9wcJA0Gg3985//JCKiCxcuEABqbGyU2hw/fpwEQaCbN29O6nsdDgcBIIfD8TDlMxbQRFGkS5cu0b///W8aHh4mURTlLkky2W10RofjuHr1Kux2O/Lz86V5er0e2dnZsFqtAACr1YqYmBhkZmZKbfLz86FQKFBfX3/fz3W73XA6nX4TY6HM6/XCarWir68Pu3btQmRkZEAcct1tRgPIbrcDAAwGg998g8EgLbPb7UhKSvJbrlQqERcXJ7W5W0lJCfR6vTSZzeaZLJuxgOLxeFBdXQ2VSoXc3FyoVCq5S5q2gBiQrLi4GA6HQ5q6urrkLomxWUdEGBsbQ2VlJVJSUpCZmRmQez13mtEAMhqNAICenh6/+T09PdIyo9GI3t5ev+VerxcDAwNSm7tpNBrodDq/ibFQQkRwu90oKyvD0qVL8dhjjwV8+AAzHEBpaWkwGo0oLy+X5jmdTtTX18NisQAALBYLBgcH0dzcLLWpqKiAKIrIzs6eyXIYCxojIyOoqKhATk4OTCZTUIQPMI0LEYeHh3HlyhXp9dWrV9Ha2oq4uDikpKRg//79+N3vfofFixcjLS0N7733HpKTk7F7924AwLJly/Dkk0/i5ZdfxscffwyPx4N9+/bhV7/6FZKTk2fshzEWDIgIw8PDKC8vx+bNmxEXFyd3STNrqqfXTp48SQDumV588UUi+v7U4HvvvUcGg4E0Gg1t376dOjo6/D6jv7+fnn/+eYqKiiKdTkcvvfQSDQ0NTboGPg3PQoXT6aTjx4/T7du359Rp9p8y2W1UILpjtOkA4XQ6odfr4XA4uD+IBSUiwuDgIE6fPo28vDzExMTIXdKUTHYbDYizYIyFEiLC6OgoTp8+DYvFAr1eL3dJjwzfjMrYHDM2NoZTp05h48aNwdfncxfeA2JsDnG73SgvL0dmZuaknywRyDiAGJsjJq5wXrt2LZKSkoLmVPuDcAAxNgd4PB6Ul5fjsccew/z580MifAAOIMZkR0RoaGiA0WhEWlpayIQPwAHEmKyICG1tbVCr1VizZk1IhQ/AAcSYbIgI58+fR3d3NzIyMkIufAAOIMZkY7PZ0N3djW3btvk9cz2UhOavZkxmEzdkb968GWq1Wu5yZMMBxNgsc7lcOHbsGCwWC7RardzlyCqgr4QOwNvYWIjz+XyoqqrCli1bkJCQIHc5sgvoPSCn08khxAKGKIqwWq0wmUw89MwPAjqAamtrMT4+LncZjP0kIsLFixcxPj6O5cuXh+QZr/sJ6ABKT09HeXk5vF6v3KUw9qOICD09Pejq6sKmTZs4fO4Q0AGUmpqKhIQEtLa28qEYm7OGh4dhtVqxefPmgH6CxaMQ0AEkCAIyMjLgdDpx/fp1DiE253g8Hpw6dQpZWVkIDw+Xu5w5J6ADCADCwsKQl5eHpqYm3L59W+5yGJOIooi6ujosWbIkpG4wnYqADyAAUKlUePzxx1FdXQ232y13OYxJt1koFAosXryYw+dHBEUACYKA+Ph4rFq1CrW1tRBFUe6SWAgjIvT19eHq1avIzs4O2dssJiNo1owgCEhLS0NkZCTOnDnD/UFMNi6XC3V1ddi2bRuUyoC+1veRC5oAAr4PofXr18Nut8Nut3MIsVnn9XpRUVGB9evXIzo6Wu5y5rygCiAAUCqV2Lp1K+rr6+FyueQuh4UQIkJTUxMSExMxf/58ucsJCEEXQAAQHh6OrKwsnDhxAh6PR+5yWAggIty4cQNDQ0PIzMzkTudJCsoAEgQBycnJSE5O5v4gNiucTicaGhqwadMmhIWFyV1OwAjKAAK+D6F169ZhaGgInZ2dcpfDgpjX60V5eTmys7Oh0WjkLiegBG0AAd9fpJibm4umpiYMDAzwnhCbcRP9PkuWLIHJZOJDrykK6gACAI1GA4vFgsrKSvh8PrnLYUHmu+++w+joKJYvXy53KQFpSgFUUlKCrKwsREdHIykpCbt370ZHR4dfm7GxMRQVFSE+Ph5RUVF47rnn0NPT49ems7MThYWFiIiIQFJSEt55551Hdke7IAiYN28eFi9ejKamJt4LYjNmcHAQDQ0NyMnJ4YsNp2lKa62yshJFRUWoq6tDWVkZPB4PduzYgZGREanNm2++ia+++gqHDx9GZWUluru78eyzz0rLfT4fCgsLMT4+jtraWnz++ef47LPPcODAgZn7VXcRBAErVqzA8PAwvvvuOw4h9tA8Hg9qamrwxBNPICIiQu5yAhc9hN7eXgJAlZWVREQ0ODhIKpWKDh8+LLW5ePEiASCr1UpERMeOHSOFQkF2u11qc/DgQdLpdOR2uyf1vQ6HgwCQw+GYUr2jo6P0j3/8g5xO55Tex9idRFGk06dPU3t7O4miKHc5c9Jkt9GH2m90OBwAgLi4OABAc3MzPB4P8vPzpTbp6elISUmB1WoFAFitVqxatQoGg0FqU1BQAKfTifb29vt+j9vthtPp9JumIzw8HPn5+aiqquJBzNi0EBG+/fZb9Pf3Iz09nTudH9K0A0gURezfvx+5ublYuXIlAMBut0OtViMmJsavrcFggN1ul9rcGT4TyyeW3U9JSQn0er00mc3m6ZaNxMREmM1m7g9i0zI0NIQLFy7giSee4H6fGTDtNVhUVITz58/j0KFDM1nPfRUXF8PhcEhTV1fXtD9LEASsXLkSfX19uHbtGocQmzSv14uqqirk5uaG/ON0Zsq0Amjfvn04evQoTp48CZPJJM03Go0YHx/H4OCgX/uenh4YjUapzd1nxSZeT7S5m0ajgU6n85sehkKhQH5+Ps6dO4fh4eGH+iwWGogIVVVVSE5O5sfpzKApBRARYd++fThy5AgqKiqQlpbmtzwjIwMqlQrl5eXSvI6ODnR2dsJisQAALBYL2tra0NvbK7UpKyuDTqeb1WspwsPDkZ2djZqaGu4PYg9EROjs7ITX68XatWu532cmTaVn+9VXXyW9Xk+nTp0im80mTaOjo1KbvXv3UkpKClVUVFBTUxNZLBayWCzScq/XSytXrqQdO3ZQa2srlZaWUmJiIhUXF894D/tPEUWRmpub6fTp03w2g/2ooaEh+uKLL8jlcsldSsCY7DY6pQACcN/p008/ldq4XC567bXXKDY2liIiImjPnj1ks9n8PufatWu0c+dOCg8Pp4SEBHr77bfJ4/HM+I+bDK/XS9988w11dnY+9Gex4DM+Pk6HDx+m7u5uuUsJKJPdRgWiwOuFdTqd0Ov1cDgcD90fBHw/gl1paSl27NiBiIgI3sVmAL4/9GpubpYGuuN/F5M32W2UzyMC0Gq12LBhA06cOMHjSTPJjRs3cPv2be73eYQ4gPB/4weZTCacPXuWT80zOJ1O1NfXIzc3l8f3eYQ4gH4gCALWrl2Lnp4e2Gw2ucthMvL5fKipqUFOTg4/TPAR4wC6Q1hYGDZv3oyamhq/G2xZ6CAitLa2Yt68efwwwVnAAXSXiIgI6fog7g8KLUSErq4u9PX1YfXq1Rw+s4AD6C6CICAlJQXx8fFoa2vj/qAQ4nK5UF9fz+M6zyIOoPuYGE/6xo0buHnzJodQCPD5fKiqqsLGjRsRGRkpdzkhgwPoR4SFhSEvLw8NDQ38fLEgR0Q4e/YskpKSkJycLHc5IYUD6AH0ej2ysrJQU1PD40kHKSJCd3c3bty4gTVr1nC/zyzjAPoJJpMJ8fHxaGlp4UOxIORyudDY2Ijt27fz+D4y4DX+EwRBwJo1a2C329Hd3c0hFES8Xi+qq6uxYcMGREZG8t6PDDiAJkGpVGL79u3cHxRE6IfnecXHx2PevHlylxOyOIAmaWL8oFOnTvH4QUGgq6sLw8PDWLduHe/5yIgDaJImni+WmJjIz5sPYESEoaEhtLS0IC8vj6/3kRkH0BRMDMvgdDr5efMByufz4dixY8jJyeHnuM8BHEBTNPG8+dbWVty+fZv3hAKIKIpoaGjAqlWrYDQa+dBrDuAAmgaNRoPc3FxUVlZyf1CAICJcuXIFXq8X6enpcpfDfsABNA2CICA+Ph4rVqyA1Wrlm1bnOCJCb28vmpqa+Dnucwz/JaZJEAQsWrQIYWFhaG9v50OxOcztdqOurg5PPfUU1Gq13OWwO3AAPQRBELBhwwZcvnyZBzGbo3w+H6qrq7F+/XpER0fLXQ67CwfQQ1KpVHjyySfR2NjIDzmcYyYGlY+JiYHJZOJO5zmIA2gGREREwGKxoLKyEh6PR+5yGL4Pn//9739wuVz8RIs5jANoBgiCgMTERKSmpqKuro47pWVGROjv78eFCxewceNGvthwDuMAmiGCIGD58uXQaDTcKS0zl8uFr7/+Gnl5eVAqlXKXwx6AA2gGCYKAzMxMdHV14dq1axxCMvB6vaiqqsK2bdsQGxvLh15zHAfQDFMoFNiyZQvOnDnDV0rPMlEUUVtbi9TUVBiNRrnLYZPAAfQIREZGIj8/HzU1NRgbG5O7nJBARGhra0NYWBiWLl3Kez4BYkoBdPDgQaxevRo6nQ46nQ4WiwXHjx+Xlo+NjaGoqAjx8fGIiorCc889h56eHr/P6OzsRGFhISIiIpCUlIR33nknKG9niI6ORkZGBkpLSzE+Pi53OUGNiHDt2jUMDAwgJyeHwyeATCmATCYTPvjgAzQ3N6OpqQnbtm3DM888g/b2dgDAm2++ia+++gqHDx9GZWUluru78eyzz0rv9/l8KCwsxPj4OGpra/H555/js88+w4EDB2b2V80BE497XrZsGWpra/nM2CNCRLDb7Whvb+fHKAciekixsbH017/+lQYHB0mlUtHhw4elZRcvXiQAZLVaiYjo2LFjpFAoyG63S20OHjxIOp2O3G73pL/T4XAQAHI4HA9b/iMniiK1tLRQU1MTiaIodzlBx+l00n//+18aHh6WuxR2h8luo9PuA/L5fDh06BBGRkZgsVjQ3NwMj8eD/Px8qU16ejpSUlJgtVoBAFarFatWrYLBYJDaFBQUwOl0SntRwWZiTOmBgQGcO3eOO6Vn0OjoKKqrq7Fp0yZ+lleAmnIAtbW1ISoqChqNBnv37sWRI0ewfPly2O12qNVqxMTE+LU3GAyw2+0AALvd7hc+E8snlv0Yt9sNp9PpNwUShUKBrVu3YmBggE/PzxC3243S0lKsWbPmnn9zLHBMOYCWLl2K1tZW1NfX49VXX8WLL76ICxcuPIraJCUlJdDr9dJkNpsf6fc9CkqlEnl5ebh48SLsdjuH0EPw+XyoqanB+vXrMX/+fO50DmBTDiC1Wo1FixYhIyMDJSUlWLNmDf70pz/BaDRifHwcg4ODfu17enqkazKMRuM9Z8UmXj/ouo3i4mI4HA5p6urqmmrZc4JKpcLjjz+OpqYm9Pf3y11OQJoIH5PJhAULFshdDntID30dkCiKcLvdyMjIgEqlQnl5ubSso6MDnZ2dsFgsAACLxYK2tjb09vZKbcrKyqDT6bB8+fIf/Q6NRiOd+p+YAlVERAS2bt2KxsZG3L59W+5yAoooimhqakJ4eDiWLFnCez7BYCo92++++y5VVlbS1atX6dy5c/Tuu++SIAj0zTffEBHR3r17KSUlhSoqKqipqYksFgtZLBbp/V6vl1auXEk7duyg1tZWKi0tpcTERCouLn4kPexzlSiKNDg4SIcOHaKBgQE+OzYJPp+PmpqaqLW1lXw+n9zlsJ8w2W10SgH061//mhYsWEBqtZoSExNp+/btUvgQEblcLnrttdcoNjaWIiIiaM+ePWSz2fw+49q1a7Rz504KDw+nhIQEevvtt8nj8UyljIAPoAn9/f1UWlpKTqdT7lLmNJ/PR7W1tXT69GkOnwAx2W1UIAq83lCn0wm9Xg+HwxHQh2MAcPv2bVRXVyMvL49vnrwPURTR1taG8fFxZGRk8HjOAWKy2yj/NWUWExMDi8WC2tpaDA0NyV3OnCKKIurq6uDxeDh8ghT/RWU2MZhZbm4uamtrMTAwwKfo8X34tLS0AADWr1/P4ROk+K86R8TExCA7OxsnTpwI+VP0Pp8PdXV1UKlU/BidIMd/2TlCEATExsZi586daGxshM1mC8k9IY/Hg8rKSmg0GqxevZrDJ8jxX3eOiYqKwtatW9HS0oIrV66EVAiNjIygoqICqampPJB8iOAAmmMEQYBWq8WOHTtgs9lw7ty5aQ/lQUS4dOkSysrKcOnSJQwMDMDlcsHn881w1Q+HiOBwOHD8+HEsW7YMaWlpHD4hgkfsnqNUKhVyc3PR0NCAU6dOYdOmTVCpVFP6DCLCJ598gg8//BARERHQ6/VIS0tDWloaFi5ciKVLlyI1NRWJiYmIj4+HVquFIAjSNBuICNevX8e5c+ewbds2xMXFzcr3srmBrwOa44gI3333HS5fvoy8vDxER0dPOhw8Hg+2bNkiDYdyN4VCAbVajbi4OCQmJmLBggVYsmQJHnvsMSxcuFAKq0f1ZAlRFNHe3o7Lly+jsLAQGo2G93yCxGS3Ud4DmuMEQcDChQuh0+lQWlqKnJwcmM3mSW2oXq/3gUOXiKKIsbExdHd3o7u7G2fPnpW+cyKYJgZ5n0lEBLfbjZqaGuh0OjzzzDP8+JwQxX1AAWDiWqFdu3aho6MDNTU1kxpHu6+vb1o3vE4EhEajQXh4+HRKfuBn9/b24ujRo0hOTkZWVhaHTwjjAAoQE53T27dvh1arxZdffolbt2498CxZV1cX7HY7wsPDkZaWhkWLFiEqKmrS37l06VIkJCTMRPkAvj8krK6uRn19PQoKCrBs2TI+5Apx/L+eAKNQKJCVlYVFixahoqICSUlJyMnJgVKpvGdj7u/vR0JCAp5++mnMmzcPgiBgYGAAR48exfXr16V2arUaOp0OPp8PDodDOusWExMzI9fhEBFsNhsaGhpgMpl48Hgm4QAKUDExMXjmmWdw7tw5fPnll8jMzITZbPYLDKvVisLCQphMJimcJgLpb3/7GxwOB+bPn4+CggIYjUZ4vV50dHSgrKwMo6Oj2Lp160PXOTo6isbGRnR3d2Pnzp3Q6/W818MkHEABShAEKJVKrFu3DosWLUJNTQ3a2tqQk5OD+Ph4CIIAs9kMtVrtt8ELgoD4+HgsWLAAV69exa5du5CUlCR1PK9duxY+nw/Hjh2DwWCYVlgQEbxeL86ePYvLly8jMzMTeXl5vNfD7sGn4YPEROduRUUFoqOjkZ2dDZfLhc8///y+FzJ2d3djZGQEixYtuidk3G43Dh8+jC+++AKrVq2aUh1utxttbW24cuUKFi5ciFWrVvHp9RDEp+FDjCAIMBgM+MUvfgGbzYbq6mqMjo5CqVTC7Xb7BUBiYiJef/11nD9/HidPnrzns9RqNQoKCiZ9+p2IMDQ0hLa2NnR1dcFkMuHpp59GREQEBw97IN4DCkL0/UiX6Ovrw9mzZ/Htt9+iv78fRISkpCQ89dRTMJlMsNls+Pvf/w6Xy+X3/gULFuD555+/5/Dtzs8HAJfLhStXruDChQtQKBRYvHgxVqxYAZVKxcET4ia7jXIABTkiwtjYGGw2G86cOYNbt27BZDIhOTkZCxYswKVLl1BdXY2xsTEIgoCkpCTs2bNH6hea+Ayv1wu3242BgQFcv34dV65cgVqthtlsRnp6OhISEvjOdSbhAGL3ICKMj4+jt7cXV65cwe3bt9Hd3Q2Xy4WbN28iPj4ey5Ytg1arhVarxeDgIBQKBUZGRiCKIrRaLSIjI7F48WIkJibO2Gl6Fny4D4jdQxAEaDQamM1mmM1mEBFEUYQoihgaGoLL5YLH44Hb7YZarQYRQafTISoqSjqsms0bVVnw4wAKYYIgICwsDGFhYXwXOpMF7z8zxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYk01A3ooxcf/sgx45wxiTz8S2+VP3ugdkAPX39wMAzGazzJUwxh5kaGgIer3+R5cHZABN3DjZ2dn5wB/H/DmdTpjNZnR1dfEwJpPE62x6JkbJTE5OfmC7gAygiTFo9Ho9/6OYBp1Ox+ttinidTd1kdg64E5oxJhsOIMaYbAIygDQaDd5//31oNBq5SwkovN6mjtfZoxWQY0IzxoJDQO4BMcaCAwcQY0w2HECMMdlwADHGZBOQAfTRRx8hNTUVWq0W2dnZaGhokLsk2ZSUlCArKwvR0dFISkrC7t270dHR4ddmbGwMRUVFiI+PR1RUFJ577jn09PT4tens7ERhYSEiIiKQlJSEd955B16vdzZ/imw++OADCIKA/fv3S/N4nc0SCjCHDh0itVpNn3zyCbW3t9PLL79MMTEx1NPTI3dpsigoKKBPP/2Uzp8/T62trfSzn/2MUlJSaHh4WGqzd+9eMpvNVF5eTk1NTZSTk0MbN26Ulnu9Xlq5ciXl5+fTmTNn6NixY5SQkEDFxcVy/KRZ1dDQQKmpqbR69Wp64403pPm8zmZHwAXQhg0bqKioSHrt8/koOTmZSkpKZKxq7ujt7SUAVFlZSUREg4ODpFKp6PDhw1KbixcvEgCyWq1ERHTs2DFSKBRkt9ulNgcPHiSdTkdut3t2f8AsGhoaosWLF1NZWRlt2bJFCiBeZ7MnoA7BxsfH0dzcjPz8fGmeQqFAfn4+rFarjJXNHQ6HA8D/3bDb3NwMj8fjt87S09ORkpIirTOr1YpVq1bBYDBIbQoKCuB0OtHe3j6L1c+uoqIiFBYW+q0bgNfZbAqom1Fv3boFn8/n90cHAIPBgEuXLslU1dwhiiL279+P3NxcrFy5EgBgt9uhVqsRExPj19ZgMMBut0tt7rdOJ5YFo0OHDqGlpQWNjY33LON1NnsCKoDYgxUVFeH8+fOoqamRu5Q5raurC2+88QbKysqg1WrlLiekBdQhWEJCAsLCwu45G9HT0wOj0ShTVXPDvn37cPToUZw8eRImk0mabzQaMT4+jsHBQb/2d64zo9F433U6sSzYNDc3o7e3F+vXr4dSqYRSqURlZSU+/PBDKJVKGAwGXmezJKACSK1WIyMjA+Xl5dI8URRRXl4Oi8UiY2XyISLs27cPR44cQUVFBdLS0vyWZ2RkQKVS+a2zjo4OdHZ2SuvMYrGgra0Nvb29UpuysjLodDosX758dn7ILNq+fTva2trQ2toqTZmZmXjhhRek/+Z1Nkvk7gWfqkOHDpFGo6HPPvuMLly4QK+88grFxMT4nY0IJa+++irp9Xo6deoU2Ww2aRodHZXa7N27l1JSUqiiooKamprIYrGQxWKRlk+cUt6xYwe1trZSaWkpJSYmhtQp5TvPghHxOpstARdARER//vOfKSUlhdRqNW3YsIHq6urkLkk2AO47ffrpp1Ibl8tFr732GsXGxlJERATt2bOHbDab3+dcu3aNdu7cSeHh4ZSQkEBvv/02eTyeWf418rk7gHidzQ4ejoMxJpuA6gNijAUXDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJ5v8DEYLBF8sQC/MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (1,), float32)\n",
      "state= [-0.42624906  0.        ]\n",
      "action= [-0.8223835]\n",
      "next_state= [-0.42820242 -0.00195336]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f54ee457eb0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=2048,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-400.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(200_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/4.PPO.Mountain Car Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(34.0, 0.5477225575051661)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/4.PPO.Mountain Car Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "414e1478",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhpklEQVR4nO3da1BU9/0/8PdZ2AsL7ILcVoQVjBpEEeUirBATlUCMaRLjg7bjtE6SSUaDbUwymQkz/5jpTGfItE96+aXmQdqYznRialvb6igRQbnIcpeIaIgmIigsqLC7XJZld8/n/yDhVIwaUODs5fOaOTPZc767+9mD551zvuec7xGIiMAYYzJQyF0AYyxwcQAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmQjWwB98MEHSEpKgkajQU5ODhobG+UqhTEmE1kC6LPPPsObb76J9957D62trUhPT0dRUREGBgbkKIcxJhNBjptRc3JykJ2djf/7v/8DAIiiiMTERPziF7/AO++8M9/lMMZkEjzfXzgxMYGWlhaUlJRI8xQKBQoKCmA2m+/6HqfTCafTKb0WRRGDg4OIioqCIAhzXjNjbGaICMPDw4iPj4dCce8DrXkPoJs3b8Lj8SAuLm7K/Li4OHz55Zd3fU9paSl+9atfzUd5jLFZ1NPTg4SEhHsun/cAehAlJSV48803pdc2mw1GoxE9PT3Q6XQyVsYYuxu73Y7ExESEh4fft928B1B0dDSCgoLQ398/ZX5/fz8MBsNd36NWq6FWq783X6fTcQAx5sV+qItk3s+CqVQqZGZmoqKiQponiiIqKipgMpnmuxzGmIxkOQR78803sXPnTmRlZWHdunX43e9+h9HRUbz44otylMMYk4ksAfTjH/8YN27cwL59+2CxWLBmzRqUlZV9r2OaMebfZLkO6GHZ7Xbo9XrYbDbuA2LMC013G/WJs2CMsbk3MXEdHs8QlMpEBAVpAQTP+XV2HECMMQCA1fpv9Pb+PwQHx0KjSYFWm46QkNXQaFZCqYxDUJAek+etZiuYOIAYY98R4fFY4fFY4XR+BZvtvwCCoVBooFIlQq1eitDQddDpihAamj0r38gBxBi7BwWCgyMQHLwQISErpT0ipTJ+1r6BA4gxBgBwuYKh0axBSMiK78JmLdTqZCiV8VAoNAAUs94nxAHEGAMAtLbGYP36/2DBgkUQhKB5+U4eEZExBgBwuRQQhLB5Cx+AA4gxJiMOIMYCFBFB7uuQuQ+IsQDjdrtx7do1HD16FBqNBi+99NJ9Bw2bSxxAjAUAIoLD4UBTUxM+/fRTHDlyBBaLBUlJSdi6dSsWLlwoS10cQIz5MY/Hg97eXhw5cgSfffYZGhsbMT4+Li3/5ptv8Pe//x2//OUvZamPA4gxPzO5t9PW1oa//vWvKCsrQ09PD0RRvGv7jz76CD/72c/mucpvcQAx5idEUURvby9OnDiBAwcOoLW1FaOjoz/4vkuXLqGmpkaWDmkOIMZ8nMvlQmNjIw4dOoTDhw/j+vXr8Hg8931PeHg4YmNj4fF40NfXhz/96U/YuXPnPFX8PxxAjPm4lpYWPPPMM7BardNqv2TJEjz99NOIjIwEEeHatWs4ceIEzpw5g6eeempui70DXwfEmI9LS0tDWlratNpGRkbimWeeQVRUFIKCghAcHIzFixdjy5Yt6OrqmttC74L3gBjzcVqtFrt374bZbIbb7ZbmpaamIjo6Gjdv3sSFCxcwNjaGJUuWIDIycspNpYIgYPHixVizZs09v+P2/iEi+t77HxQHEGM+ThAEPP3000hNTcW5c+eg1Wqxfft2JCcnS+GQmpqKf/7zn/fsaBYEQWpLRLDb7RgZGUFfXx9u3LiB8fFx2Gw2hISEYGRkBCEhIVAqldBoNFiwYAHCwsKwaNEihIeHQ6VSTbt2DiDG/IBOp8P27dvR3t6O/Px8LFmyZMqeSXJyMrZt24bc3Fy4XC7YbDZpGRFh0aJFcDgc+PzzzyGKIhwOBwwGA2JiYrB06VJERkZCqVROuWJaFEWMj49jcHAQfX19uHLlCmw2G0JDQxEZGTmtujmAGPMDgiDg5z//Of785z/DaDR+77BIEARs2LABL7/8Mq5evYpjx45hcHAQLpcLRITR0VFEREQgMzMTRqNxyoNA73eIpdfrERcXh5SUFGme1WpFZ2fntOrmAGLMTxiNRuzYsQM9PT13Xa7VagEACxcuREZGBk6ePIn169cjPT0d0dHRCA5+8EHob39fZGQkUlNTp/c+fiwPY/7j+vXraG5uxldffYWxsTFpfkhICLZt2wa73Y7z588jJSUFK1euRFhY2JzUMd1tlAOIMT9DRLh06RLOnDmDoaEh6HQ66PV6OJ1OpKamYtWqVVAqlXP6yB1+LhhjAUoQBCxbtgxJSUno6urC2bNnYTAYkJGRAa1WO+fP+poJDiDG/NDk7Rn9/f0oLCxERESEVwXPJA4gxvwIEaG3txenTp1CVlYWTCYTgoLmb4znmeIAYsxPeDwetLW1ob+/H88++yzCw8O9cq/ndjO+F6y6uho/+tGPEB8fD0EQ8O9//3vKciLCvn37sHDhQoSEhKCgoACXLl2a0mZwcBA7duyATqdDREQEXn75ZYyMjDzUD2EskI2OjuLzzz+HIAgoKiqCTqfz+vABHiCARkdHkZ6ejg8++OCuy3/zm9/gD3/4Az788EM0NDQgNDQURUVFU0Zh27FjBzo6OlBeXo6jR4+iuroar7766oP/CsYCFBGhp6cHJ0+exNq1a7F27VqvPuT6HnoIAOjw4cPSa1EUyWAw0G9/+1tpntVqJbVaTZ9++ikREV24cIEAUFNTk9Tm+PHjJAgCXb9+fVrfa7PZCADZbLaHKZ8xnyaKIn355Zf0j3/8g0ZGRkgURblLkkx3G53V4TiuXLkCi8WCgoICaZ5er0dOTg7MZjMAwGw2IyIiAllZWVKbgoICKBQKNDQ03PVznU4n7Hb7lImxQOZ2u2E2m3Hjxg08++yzCA0N9YlDrjvNagBZLBYAQFxc3JT5cXFx0jKLxYLY2Ngpy4ODg7FgwQKpzZ1KS0uh1+ulKTExcTbLZsynuFwu1NTUQKlUIi8vD0qlUu6SHphPDEhWUlICm80mTfe614Uxf0ZEGB8fR1VVFYxGI7Kysnxyr+d2sxpABoMBANDf3z9lfn9/v7TMYDBgYGBgynK3243BwUGpzZ3UajV0Ot2UibFAQkRwOp0oLy/Ho48++r3hNnzVrAZQcnIyDAYDKioqpHl2ux0NDQ0wmUwAAJPJBKvVipaWFqlNZWUlRFFETk7ObJbDmN8YHR1FZWUlcnNzkZCQ4BfhAzzAhYgjIyO4fPmy9PrKlStoa2vDggULYDQasXfvXvz617/GsmXLkJycjHfffRfx8fF4/vnnAQArVqzAU089hVdeeQUffvghXC4X9uzZg5/85CeIj4+ftR/GmD8gIoyMjKCiogIbNmzAggUL5C5pds309NqpU6cIwPemnTt3EtG3pwbfffddiouLI7VaTZs3b6bOzs4pn3Hr1i366U9/SmFhYaTT6ejFF1+k4eHhadfAp+FZoLDb7XT8+HEaGhryqtPsP2S62ygPx8GYFyIiWK1WnDlzBvn5+YiIiJC7pBmZ7jbqE2fBGAskRISxsTGcOXMGJpMJer1e7pLmDN+MypiXGR8fx+nTp7F+/Xr/6/O5A+8BMeZFnE4nKioqkJWVNe0nS/gyDiDGvMTkFc5r1qxBbGys35xqvx8OIMa8gMvlQkVFBZYsWYJFixYFRPgAHECMyY6I0NjYCIPBMOVppoGAA4gxGRER2tvboVKpkJ6eHlDhA3AAMSYbIsL58+fR29uLzMzMgAsfgAOIMdn09fWht7cXmzZtmvLM9UASmL+aMZlN3pC9YcMGqFQqucuRDQcQY/PM4XDg2LFjMJlM0Gg0cpcjK5++EtoHb2NjAc7j8aC6uhqPP/44oqOj5S5Hdj69B2S32zmEmM8QRRFmsxkJCQk89Mx3fDqA6urqMDExIXcZjP0gIsLFixcxMTGB1NTUgDzjdTc+HUApKSmoqKiA2+2WuxTG7omI0N/fj56eHjz22GMcPrfx6QBKSkpCdHQ02tra+FCMea2RkRGYzWZs2LDBp59gMRd8OoAEQUBmZibsdjuuXr3KIcS8jsvlwunTp5GdnY2QkBC5y/E6Ph1AABAUFIT8/Hw0NzdjaGhI7nIYk4iiiPr6eixfvjygbjCdCZ8PIABQKpV44oknUFNTA6fTKXc5jEm3WSgUCixbtozD5x78IoAEQUBUVBTS0tJQV1cHURTlLokFMCLCjRs3cOXKFeTk5ATsbRbT4TdrRhAEJCcnIzQ0FGfPnuX+ICYbh8OB+vp6bNq0CcHBPn2t75zzmwACvg2hjIwMWCwWWCwWDiE279xuNyorK5GRkYHw8HC5y/F6fhVAABAcHIyNGzeioaEBDodD7nJYACEiNDc3IyYmBosWLZK7HJ/gdwEEACEhIcjOzsbJkyfhcrnkLocFACLCtWvXMDw8jKysLO50nia/DCBBEBAfH4/4+HjuD2Lzwm63o7GxEY899hiCgoLkLsdn+GUAAd+G0Nq1azE8PIzu7m65y2F+zO12o6KiAjk5OVCr1XKX41P8NoCAby9SzMvLQ3NzMwYHB3lPiM26yX6f5cuXIyEhgQ+9ZsivAwgA1Go1TCYTqqqq4PF45C6H+Zmvv/4aY2NjSE1NlbsUnzSjACotLUV2djbCw8MRGxuL559/Hp2dnVPajI+Po7i4GFFRUQgLC8P27dvR398/pU13dze2bt0KrVaL2NhYvP3223N2R7sgCFi4cCGWLVuG5uZm3gtis8ZqtaKxsRG5ubl8seEDmtFaq6qqQnFxMerr61FeXg6Xy4XCwkKMjo5Kbd544w0cOXIEhw4dQlVVFXp7e/HCCy9Iyz0eD7Zu3YqJiQnU1dXhk08+wYEDB7Bv377Z+1V3EAQBK1euxMjICL7++msOIfbQXC4Xamtr8eSTT0Kr1cpdju+ihzAwMEAAqKqqioiIrFYrKZVKOnTokNTm4sWLBIDMZjMRER07dowUCgVZLBapzf79+0mn05HT6ZzW99psNgJANpttRvWOjY3R3/72N7Lb7TN6H2O3E0WRzpw5Qx0dHSSKotzleKXpbqMPtd9os9kAAAsWLAAAtLS0wOVyoaCgQGqTkpICo9EIs9kMADCbzUhLS0NcXJzUpqioCHa7HR0dHXf9HqfTCbvdPmV6ECEhISgoKEB1dTUPYsYeCBHh0qVLuHXrFlJSUrjT+SE9cACJooi9e/ciLy8Pq1atAgBYLBaoVCpERERMaRsXFweLxSK1uT18JpdPLrub0tJS6PV6aUpMTHzQshETE4PExETuD2IPZHh4GBcuXMCTTz7J/T6z4IHXYHFxMc6fP4+DBw/OZj13VVJSApvNJk09PT0P/FmCIGDVqlW4ceMGurq6OITYtLndblRXVyMvLy/gH6czWx4ogPbs2YOjR4/i1KlTSEhIkOYbDAZMTEzAarVOad/f3w+DwSC1ufOs2OTryTZ3UqvV0Ol0U6aHoVAoUFBQgHPnzmFkZOShPosFBiJCdXU14uPj+XE6s2hGAURE2LNnDw4fPozKykokJydPWZ6ZmQmlUomKigppXmdnJ7q7u2EymQAAJpMJ7e3tGBgYkNqUl5dDp9PN67UUISEhyMnJQW1tLfcHsfsiInR3d8PtdmPNmjXc7zObZtKzvXv3btLr9XT69Gnq6+uTprGxManNrl27yGg0UmVlJTU3N5PJZCKTySQtd7vdtGrVKiosLKS2tjYqKyujmJgYKikpmfUe9h8iiiK1tLTQmTNn+GwGu6fh4WH617/+RQ6HQ+5SfMZ0t9EZBRCAu04ff/yx1MbhcNBrr71GkZGRpNVqadu2bdTX1zflc7q6umjLli0UEhJC0dHR9NZbb5HL5Zr1HzcdbrebTpw4Qd3d3Q/9Wcz/TExM0KFDh6i3t1fuUnzKdLdRgcj3emHtdjv0ej1sNttD9wcB345gV1ZWhsLCQmi1Wt7FZgC+PfRqaWmRBrrjfxfTN91tlM8jAtBoNFi3bh1OnjzJ40kzybVr1zA0NMT9PnOIAwj/Gz8oISEBX3zxBZ+aZ7Db7WhoaEBeXh6P7zOHOIC+IwgC1qxZg/7+fvT19cldDpORx+NBbW0tcnNz+WGCc4wD6DZBQUHYsGEDamtrp9xgywIHEaGtrQ0LFy7khwnOAw6gO2i1Wun6IO4PCixEhJ6eHty4cQOrV6/m8JkHHEB3EAQBRqMRUVFRaG9v5/6gAOJwONDQ0MDjOs8jDqC7mBxP+tq1a7h+/TqHUADweDyorq7G+vXrERoaKnc5AYMD6B6CgoKQn5+PxsZGfr6YnyMifPHFF4iNjUV8fLzc5QQUDqD70Ov1yM7ORm1tLY8n7aeICL29vbh27RrS09O532eecQD9gISEBERFRaG1tZUPxfyQw+FAU1MTNm/ezOP7yIDX+A8QBAHp6emwWCzo7e3lEPIjbrcbNTU1WLduHUJDQ3nvRwYcQNMQHByMzZs3c3+QH6HvnucVFRWFhQsXyl1OwOIAmqbJ8YNOnz7N4wf5gZ6eHoyMjGDt2rW85yMjDqBpmny+WExMDD9v3ocREYaHh9Ha2or8/Hy+3kdmHEAzMDksg91u5+fN+yiPx4Njx44hNzeXn+PuBTiAZmjyefNtbW0YGhriPSEfIooiGhsbkZaWBoPBwIdeXoAD6AGo1Wrk5eWhqqqK+4N8BBHh8uXLcLvdSElJkbsc9h0OoAcgCAKioqKwcuVKmM1mvmnVyxERBgYG0NzczM9x9zL8l3hAgiBg6dKlCAoKQkdHBx+KeTGn04n6+no888wzUKlUcpfDbsMB9BAEQcC6devw1Vdf8SBmXsrj8aCmpgYZGRkIDw+Xuxx2Bw6gh6RUKvHUU0+hqamJH3LoZSYHlY+IiEBCQgJ3OnshDqBZoNVqYTKZUFVVBZfLJXc5DN+GzzfffAOHw8FPtPBiHECzQBAExMTEICkpCfX19dwpLTMiwq1bt3DhwgWsX7+eLzb0YhxAs0QQBKSmpkKtVnOntMwcDgc+//xz5OfnIzg4WO5y2H1wAM0iQRCQlZWFnp4edHV1cQjJwO12o7q6Gps2bUJkZCQfenk5DqBZplAo8Pjjj+Ps2bN8pfQ8E0URdXV1SEpKgsFgkLscNg0cQHMgNDQUBQUFqK2txfj4uNzlBAQiQnt7O4KCgvDoo4/yno+PmFEA7d+/H6tXr4ZOp4NOp4PJZMLx48el5ePj4yguLkZUVBTCwsKwfft29Pf3T/mM7u5ubN26FVqtFrGxsXj77bf98naG8PBwZGZmoqysDBMTE3KX49eICF1dXRgcHERubi6Hjw+ZUQAlJCTg/fffR0tLC5qbm7Fp0yY899xz6OjoAAC88cYbOHLkCA4dOoSqqir09vbihRdekN7v8XiwdetWTExMoK6uDp988gkOHDiAffv2ze6v8gKTj3tesWIF6urq+MzYHCEiWCwWdHR08GOUfRE9pMjISProo4/IarWSUqmkQ4cOScsuXrxIAMhsNhMR0bFjx0ihUJDFYpHa7N+/n3Q6HTmdzml/p81mIwBks9ketvw5J4oitba2UnNzM4miKHc5fsdut9N//vMfGhkZkbsUdpvpbqMP3Afk8Xhw8OBBjI6OwmQyoaWlBS6XCwUFBVKblJQUGI1GmM1mAIDZbEZaWhri4uKkNkVFRbDb7dJelL+ZHFN6cHAQ586d407pWTQ2Noaamho89thj/CwvHzXjAGpvb0dYWBjUajV27dqFw4cPIzU1FRaLBSqVChEREVPax8XFwWKxAAAsFsuU8JlcPrnsXpxOJ+x2+5TJlygUCmzcuBGDg4N8en6WOJ1OlJWVIT09/Xv/5pjvmHEAPfroo2hra0NDQwN2796NnTt34sKFC3NRm6S0tBR6vV6aEhMT5/T75kJwcDDy8/Nx8eJFWCwWDqGH4PF4UFtbi4yMDCxatIg7nX3YjANIpVJh6dKlyMzMRGlpKdLT0/H73/8eBoMBExMTsFqtU9r39/dL12QYDIbvnRWbfH2/6zZKSkpgs9mkqaenZ6ZlewWlUoknnngCzc3NuHXrltzl+KTJ8ElISMDixYvlLoc9pIe+DkgURTidTmRmZkKpVKKiokJa1tnZie7ubphMJgCAyWRCe3s7BgYGpDbl5eXQ6XRITU2953eo1Wrp1P/k5Ku0Wi02btyIpqYmDA0NyV2OTxFFEc3NzQgJCcHy5ct5z8cfzKRn+5133qGqqiq6cuUKnTt3jt555x0SBIFOnDhBRES7du0io9FIlZWV1NzcTCaTiUwmk/R+t9tNq1atosLCQmpra6OysjKKiYmhkpKSOelh91aiKJLVaqWDBw/S4OAgnx2bBo/HQ83NzdTW1kYej0fuctgPmO42OqMAeumll2jx4sWkUqkoJiaGNm/eLIUPEZHD4aDXXnuNIiMjSavV0rZt26ivr2/KZ3R1ddGWLVsoJCSEoqOj6a233iKXyzWTMnw+gCbdunWLysrKyG63y12KV/N4PFRXV0dnzpzh8PER091GBSLf6w212+3Q6/Ww2Ww+fTgGAENDQ6ipqUF+fj7fPHkXoiiivb0dExMTyMzM5PGcfcR0t1H+a8osIiICJpMJdXV1GB4elrscryKKIurr6+FyuTh8/BT/RWU2OZhZXl4e6urqMDg4yKfo8W34tLa2AgAyMjI4fPwU/1W9REREBHJycnDy5MmAP0Xv8XhQX18PpVLJj9Hxc/yX9RKCICAyMhJbtmxBU1MT+vr6AnJPyOVyoaqqCmq1GqtXr+bw8XP81/UyYWFh2LhxI1pbW3H58uWACqHR0VFUVlYiKSmJB5IPEBxAXkYQBGg0GhQWFqKvrw/nzp3z+6E8iAg2mw3Hjx/HihUrkJyczOETIDiAvJRSqUReXh7GxsZw+vRpv33cDxHh6tWrqKqqwqZNm2A0Gjl8AggHkBcLCgpCbm4ujEYjysvLYbfb/eqQTBRFnD9/Hi0tLSgsLERkZKTcJbF5xgHk5QRBwCOPPIKsrCyUlZWhp6fH50OIiDA+Po7Kyko4HA4899xz0Gg0vOcTgDiAfMDktULPPvssOjs7UVtb67PjaBMRBgYGcPToUcTHxyM7O5uf3RXAOIB8xGTn9ObNm6HRaPDf//4XN2/e9Km9IZfLhZqaGjQ0NKCoqAgrVqzgvZ4Ax//r8TEKhQLZ2dlYunQpKisrERsbi9zcXAQHB3vtxkxE6OvrQ2NjIxISEnjweCbhm1F9FBHB4/Hg3LlzuHLlCrKyspCYmOh1F+6NjY2hqakJvb292LJlC/R6vdcGJZs9091GOYB8HBFheHgYtbW1EEURubm5iIqKknUjJyK43W588cUX+Oqrr5CVlYVHHnmE93oCCAdQgJns3K2srER4eDhycnIQHR0970HkdDrR3t6Oy5cv45FHHkFaWhrUajXv9QQYDqAA5fF4pP4Wt9uNdevWYdGiRXPaRzS5F9be3o6enh4kJCRg7dq10Gq1HDwBigMogNG3I13ixo0b0t7IihUrsHz5csTGxkKhUDxUMEz+k3E4HLh8+TIuXLgAhUKBZcuWYeXKlVAqlRw8AY4DiAH4NiycTie6u7vR1tYGh8OB6OhoxMfHY/HixQgJCZEOke4VGpN9Ok6nE4ODg7h69SouX74MlUqFxMREpKSkIDo62us6wJl8OIDY9xARJiYmMDAwgMuXL2NoaAgWiwVhYWEQBAHh4eHQaDRwu93QaDSwWq1QKBQYHR2FKIrQaDQIDQ3FsmXLEBMTg4iICA4ddlfT3Ub5OqAAIggC1Go1EhMTkZiYCCKCKIoQRRHDw8NwOBxwuVxwOp1QqVQgIuh0OoSFhUmHVffbU2JspjiAApggCAgKCkJQUBAWLFggdzksAPH+M2NMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZ+OStGJP3z9rtdpkrYYzdzeS2+UP3uvtkAN26dQsAkJiYKHMljLH7GR4ehl6vv+dynwygyRsnu7u77/vj2FR2ux2JiYno6enhYUymidfZg5kcJTM+Pv6+7XwygCbHoNHr9fyP4gHodDpebzPE62zmprNzwJ3QjDHZcAAxxmTjkwGkVqvx3nvvQa1Wy12KT+H1NnO8zuaWT44JzRjzDz65B8QY8w8cQIwx2XAAMcZkwwHEGJONTwbQBx98gKSkJGg0GuTk5KCxsVHukmRTWlqK7OxshIeHIzY2Fs8//zw6OzuntBkfH0dxcTGioqIQFhaG7du3o7+/f0qb7u5ubN26FVqtFrGxsXj77bfhdrvn86fI5v3334cgCNi7d680j9fZPCEfc/DgQVKpVPSXv/yFOjo66JVXXqGIiAjq7++XuzRZFBUV0ccff0znz5+ntrY2evrpp8loNNLIyIjUZteuXZSYmEgVFRXU3NxMubm5tH79emm52+2mVatWUUFBAZ09e5aOHTtG0dHRVFJSIsdPmleNjY2UlJREq1evptdff12az+tsfvhcAK1bt46Ki4ul1x6Ph+Lj46m0tFTGqrzHwMAAAaCqqioiIrJaraRUKunQoUNSm4sXLxIAMpvNRER07NgxUigUZLFYpDb79+8nnU5HTqdzfn/APBoeHqZly5ZReXk5Pf7441IA8TqbPz51CDYxMYGWlhYUFBRI8xQKBQoKCmA2m2WszHvYbDYA/7tht6WlBS6Xa8o6S0lJgdFolNaZ2WxGWloa4uLipDZFRUWw2+3o6OiYx+rnV3FxMbZu3Tpl3QC8zuaTT92MevPmTXg8nil/dACIi4vDl19+KVNV3kMURezduxd5eXlYtWoVAMBisUClUiEiImJK27i4OFgsFqnN3dbp5DJ/dPDgQbS2tqKpqel7y3idzR+fCiB2f8XFxTh//jxqa2vlLsWr9fT04PXXX0d5eTk0Go3c5QQ0nzoEi46ORlBQ0PfORvT398NgMMhUlXfYs2cPjh49ilOnTiEhIUGabzAYMDExAavVOqX97evMYDDcdZ1OLvM3LS0tGBgYQEZGBoKDgxEcHIyqqir84Q9/QHBwMOLi4nidzROfCiCVSoXMzExUVFRI80RRREVFBUwmk4yVyYeIsGfPHhw+fBiVlZVITk6esjwzMxNKpXLKOuvs7ER3d7e0zkwmE9rb2zEwMCC1KS8vh06nQ2pq6vz8kHm0efNmtLe3o62tTZqysrKwY8cO6b95nc0TuXvBZ+rgwYOkVqvpwIEDdOHCBXr11VcpIiJiytmIQLJ7927S6/V0+vRp6uvrk6axsTGpza5du8hoNFJlZSU1NzeTyWQik8kkLZ88pVxYWEhtbW1UVlZGMTExAXVK+fazYES8zuaLzwUQEdEf//hHMhqNpFKpaN26dVRfXy93SbIBcNfp448/lto4HA567bXXKDIykrRaLW3bto36+vqmfE5XVxdt2bKFQkJCKDo6mt566y1yuVzz/Gvkc2cA8TqbHzwcB2NMNj7VB8QY8y8cQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJPN/wdbNCBfwVG9ZQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34.0 67 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
